# -*- coding:utf-8 -*-
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.api import urlfetch
from google.appengine.api import memcache
import extractcontent
import urllib2
import codecs
import cgi
import unicodedata
import re
import chardet

class MainPage(webapp.RequestHandler):
  def get(self, dummy):
    self.response.headers['Content-Type'] = 'text/plain; charset=UTF-8'
    url = self.request.path_info
    url = re.sub("^\/", "", url)
    html = urlfetch.fetch(url, deadline=10).content

    extractor = extractcontent.ExtractContent()
    # オプション値を指定する
    opt = {"threshold":50}
    extractor.set_default(opt)
    resultBody = memcache.get("url:"+url)
    if resultBody is not None:
      body = resultBody
    else:
      encode = chardet.detect(html)
      html = unicode(html, encode["encoding"], "ignore")
      resultBody, title = extractor.analyse(html)
      memcache.add("url:"+url, resultBody, 100)
      body = resultBody
    self.response.out.write(body)

application = webapp.WSGIApplication([
                                      ('/(.*)', MainPage),
                                     ],
                                     debug=True)

def main():
    run_wsgi_app(application)

if __name__ == "__main__":
    main()
